<!DOCTYPE html>
<html lang="zh-cn">
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  <title>Hadoop&#43;Hive环境搭建 - macdfree&#39;s blog</title>
  <meta name="renderer" content="webkit" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>

<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />

<meta name="theme-color" content="#f8f5ec" />
<meta name="msapplication-navbutton-color" content="#f8f5ec">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="#f8f5ec">


<meta name="author" content="macd" />
  <meta name="description" content="Hadoop&#43;Hive环境搭建过程说明及注意事项
" />

  <meta name="keywords" content="blog, macd, even" />


<meta name="baidu-site-verification" content="Ak74Sk61bk" />
<meta name="google-site-verification" content="fAeFHpkrhfFS8LxdzXy0EtAPcVStv0faGn238sRF3L8" />


<meta name="generator" content="Hugo 0.54.0" />


<link rel="canonical" href="/wiki/bigdata/hadoop&#43;hive%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/" />

<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/manifest.json">
<link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">




<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>


<link href="/dist/even.min.css?v=3.1.1" rel="stylesheet">
<link href="/lib/fancybox/jquery.fancybox-3.1.20.min.css" rel="stylesheet">




<meta property="og:title" content="Hadoop&#43;Hive环境搭建" />
<meta property="og:description" content="Hadoop&#43;Hive环境搭建过程说明及注意事项" />
<meta property="og:type" content="article" />
<meta property="og:url" content="/wiki/bigdata/hadoop&#43;hive%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/" />
<meta property="article:published_time" content="2017-10-12T22:10:56&#43;08:00"/>
<meta property="article:modified_time" content="2017-10-12T22:10:56&#43;08:00"/>

<meta itemprop="name" content="Hadoop&#43;Hive环境搭建">
<meta itemprop="description" content="Hadoop&#43;Hive环境搭建过程说明及注意事项">


<meta itemprop="datePublished" content="2017-10-12T22:10:56&#43;08:00" />
<meta itemprop="dateModified" content="2017-10-12T22:10:56&#43;08:00" />
<meta itemprop="wordCount" content="1288">



<meta itemprop="keywords" content="hadoop,hive," />
<meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="Hadoop&#43;Hive环境搭建"/>
<meta name="twitter:description" content="Hadoop&#43;Hive环境搭建过程说明及注意事项"/>

<!--[if lte IE 9]>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/classlist/1.1.20170427/classList.min.js"></script>
<![endif]-->

<!--[if lt IE 9]>
  <script src="https://cdn.jsdelivr.net/npm/html5shiv@3.7.3/dist/html5shiv.min.js"></script>
  <script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->

</head>
<body>
  <div id="mobile-navbar" class="mobile-navbar">
  <div class="mobile-header-logo">
    <a href="/" class="logo">macdfree&#39;s blog</a>
  </div>
  <div class="mobile-navbar-icon">
    <span></span>
    <span></span>
    <span></span>
  </div>
</div>
<nav id="mobile-menu" class="mobile-menu slideout-menu">
  <ul class="mobile-menu-list">
    <a href="/">
        <li class="mobile-menu-item">首页</li>
      </a><a href="/categories/">
        <li class="mobile-menu-item">分类</li>
      </a><a href="/tags/">
        <li class="mobile-menu-item">标签</li>
      </a><a href="/archive/">
        <li class="mobile-menu-item">归档</li>
      </a>
  </ul>
</nav>
  <div class="container" id="mobile-panel">
    <header id="header" class="header">
        <div class="logo-wrapper">
  <a href="/" class="logo">macdfree&#39;s blog</a>
</div>

<nav class="site-navbar">
  <ul id="menu" class="menu">
    <li class="menu-item">
        <a class="menu-item-link" href="/">首页</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/categories/">分类</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/tags/">标签</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/archive/">归档</a>
      </li>
  </ul>
</nav>
    </header>

    <main id="main" class="main">
      <div class="content-wrapper">
        <div id="content" class="content">
          <article class="post">
    
    <header class="post-header">
      <h1 class="post-title">Hadoop&#43;Hive环境搭建</h1>

      <div class="post-meta">
        <span class="post-time"> 2017-10-12 </span>
        <div class="post-category">
          
            <a href="/categories/bigdata/"> BigData </a>
          
        </div>
        <span class="more-meta"> 约 1288 字 </span>
        <span class="more-meta"> 预计阅读 3 分钟 </span>
        <span id="busuanzi_container_page_pv" class="more-meta"> <span id="busuanzi_value_page_pv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 次阅读 </span>
      </div>
    </header>

    
    
<div class="post-toc" id="post-toc">
  <h2 class="post-toc-title">文章目录</h2>
  
  <div class="post-toc-content always-active">
    <nav id="TableOfContents">
<ul>
<li><a href="#安装包版本说明">安装包版本说明</a></li>
<li><a href="#安装hadoop">安装Hadoop</a>
<ul>
<li><a href="#创建用户hadoop">创建用户hadoop</a></li>
<li><a href="#修改hostname">修改hostname</a></li>
<li><a href="#设置免登录">设置免登录</a></li>
<li><a href="#添加环境变量">添加环境变量</a></li>
<li><a href="#修改配置文件">修改配置文件</a></li>
</ul></li>
<li><a href="#安装hive">安装Hive</a></li>
</ul>
</nav>
  </div>
</div>

    
    <div class="post-content">
      <p><code>Hadoop</code>+<code>Hive</code>环境搭建过程说明及注意事项</p>

<h1 id="安装包版本说明">安装包版本说明</h1>

<table>
<thead>
<tr>
<th>安装包</th>
<th>版本</th>
</tr>
</thead>

<tbody>
<tr>
<td>Hadoop</td>
<td>2.8.1</td>
</tr>

<tr>
<td>Hive</td>
<td>2.3.0</td>
</tr>
</tbody>
</table>

<h1 id="安装hadoop">安装Hadoop</h1>

<h2 id="创建用户hadoop">创建用户hadoop</h2>

<pre><code class="language-sh"># 创建hadoop用户
Shell&gt; useradd -m hadoop -s /bin/bash
# 设置密码
Shell&gt; passwd hadoop
</code></pre>

<h2 id="修改hostname">修改hostname</h2>

<pre><code class="language-sh"># 修改hostname
Shell&gt; vim /etc/hosts
</code></pre>

<p><img src="/images/clipboard.png" alt="hosts" /></p>

<p>修改后重启，使用hadoop用户登录</p>

<h2 id="设置免登录">设置免登录</h2>

<pre><code class="language-sh"># 生成密钥对
Shell&gt; ssh-keygen -t rsa
# 追加到authorized_keys中
Shell&gt; cat ./id_rsa.pub &gt;&gt; ./authorized_keys
# 设置权限
Shell&gt; chmod 644 authorized_keys
# 将下载好的hadoop和hive安装包解压缩到opt目录下
Shell&gt; tar -zxvf filename
</code></pre>

<h2 id="添加环境变量">添加环境变量</h2>

<pre><code class="language-sh">Shell&gt; vim .bashrc
</code></pre>

<p>追加如下内容</p>

<pre><code class="language-sh">export JAVA_HOME=/usr/lib/jvm/java
export HADOOP_HOME=/opt/hadoop
export HIVE_HOME=/opt/hive2
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HIVE_HOME/bin
</code></pre>

<p>使修改生效</p>

<pre><code class="language-sh">Shell&gt; source .bashrc
</code></pre>

<h2 id="修改配置文件">修改配置文件</h2>

<p>修改$HADOOP_HOME/etc/hadoop/core-site.xml</p>

<pre><code class="language-xml">&lt;configuration&gt;
    &lt;property&gt;
        &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;
        &lt;value&gt;file:/opt/hadoop/tmp&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;fs.defaultFS&lt;/name&gt;
        &lt;value&gt;hdfs://hadoop001:9000&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;hadoop.proxyuser.hadoop.hosts&lt;/name&gt;
        &lt;value&gt;*&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;hadoop.proxyuser.hadoop.groups&lt;/name&gt;
        &lt;value&gt;*&lt;/value&gt;
    &lt;/property&gt;
&lt;/configuration&gt;
</code></pre>

<p>注意<code>hadoop.tmp.dir</code>配置，防止系统重启后被操作系统删除，配置<code>fs.defaultFS</code>为指定HDFS的namenode</p>

<p>修改$HADOOP_HOME/etc/hadoop/hdfs-site.xml</p>

<pre><code class="language-xml">&lt;configuration&gt;
    &lt;property&gt;
        &lt;name&gt;dfs.replication&lt;/name&gt;
        &lt;value&gt;1&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;dfs.namenode.name.dir&lt;/name&gt;
        &lt;value&gt;file:/home/hadoop/data/dfs/name&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;dfs.datanode.data.dir&lt;/name&gt;
        &lt;value&gt;file:/home/hadoop/data/dfs/data&lt;/value&gt;
    &lt;/property&gt;
&lt;/configuration&gt;
</code></pre>

<p>配置项<code>dfs.replication</code>为HDFS的副本数，由于使用伪分布式，所以设置为1；配置项<code>dfs.namenode.name.dir</code>和<code>dfs.datanode.data.dir</code>分别为设置namenode和datanode的存储路径</p>

<p>修改$HADOOP_HOME/etc/hadoop/mapred-site.xml</p>

<pre><code class="language-xml">&lt;configuration&gt;
    &lt;property&gt;
        &lt;name&gt;mapreduce.framework.name&lt;/name&gt;
        &lt;value&gt;yarn&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;mapreduce.jobhistory.address&lt;/name&gt;
        &lt;value&gt;hadoop001:10020&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;mapreduce.jobhistory.webapp.address&lt;/name&gt;
        &lt;value&gt;hadoop001:19888&lt;/value&gt;
    &lt;/property&gt;
&lt;/configuration&gt;
</code></pre>

<p>修改$HADOOP_HOME/etc/hadoop/yarn-site.xml</p>

<pre><code class="language-xml">&lt;configuration&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.nodemanager.aux-services&lt;/name&gt;
        &lt;value&gt;mapreduce_shuffle&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.nodemanager.aux-services.mapreduce.shuffle.class&lt;/name&gt;
        &lt;value&gt;org.apache.hadoop.mapred.ShuffleHandler&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.resourcemanager.address&lt;/name&gt;
        &lt;value&gt;hadoop001:8032&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.resourcemanager.scheduler.address&lt;/name&gt;
        &lt;value&gt;hadoop001:8030&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.resourcemanager.resource-tracker.address&lt;/name&gt;
        &lt;value&gt;hadoop001:8035&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.resourcemanager.admin.address&lt;/name&gt;
        &lt;value&gt;hadoop001:8033&lt;/value&gt;
    &lt;/property&gt;
    &lt;property&gt;
        &lt;name&gt;yarn.resourcemanager.webapp.address&lt;/name&gt;
        &lt;value&gt;hadoop001:8088&lt;/value&gt;
    &lt;/property&gt;
&lt;/configuration&gt;
</code></pre>

<p>格式化namenode</p>

<pre><code class="language-sh">Shell&gt; hdfs namenode –format
</code></pre>

<p>启动服务</p>

<pre><code class="language-sh">#（在$HADOOP/sbin下）
Shell&gt; ./start-dfs.sh
Shell&gt; ./start-yarn.sh
# 或者
Shell&gt; ./start-all.sh
</code></pre>

<p>WebUI管理界面</p>

<p><code>http://192.168.131.21:50070</code>     HDFS管理界面</p>

<p><code>http://192.168.131.21:8088</code>       MR管理界面</p>

<h1 id="安装hive">安装Hive</h1>

<p>解压Hive到opt目录下</p>

<p>添加环境变量HIVE_HOME和PATH</p>

<p>添加Hive相关目录</p>

<pre><code class="language-sh">Shell&gt; hadoop fs -mkdir       /tmp
Shell&gt; hadoop fs -mkdir       /user/hive/warehouse
Shell&gt; hadoop fs -chmod g+w   /tmp
Shell&gt; hadoop fs -chmod g+w   /user/hive/warehouse
</code></pre>

<p>复制一个hive-site.xml（$HIVE_HOME/conf）</p>

<pre><code class="language-sh">Shell&gt; cp hive-default.xml.template hive-site.xml
</code></pre>

<p>修改hive.site.xml文件</p>

<ol>
<li><p>使用MySQL存储元数据</p>

<p>MySQL驱动：javax.jdo.option.ConnectionDriverName
com.mysql.jdbc.Driver
数据库连接字符串：javax.jdo.option.ConnectionURL
jdbc:mysql://192.168.131.1:3306/metastore_db?createDatabaseIfNotExist=true
数据库连接用户名：javax.jdo.option.ConnectionUserName
数据库连接密码：javax.jdo.option.ConnectionPassword
需要将MySQL的驱动添加到hive的lib下</p></li>

<li><p>设置hive用户名密码</p>

<p>用户名：hive.server2.keystore.path
密码：hive.server2.keystore.password
最好设置为hadoop，否则会有权限问题</p></li>

<li><p>替换变量</p>

<p>主要是${system:java.io.tmpdir}和${system:user.name}两个，需要修改为真实的绝对路径，例如：/opt/hive2/root/</p></li>
</ol>

<p>初始化数据库</p>

<pre><code class="language-sh">Shell&gt; schematool -dbType mysql -initSchema
</code></pre>

<p>启动服务</p>

<pre><code class="language-sh">Shell&gt; hiveserver2
</code></pre>

<p>连接hive数据库</p>

<pre><code class="language-sh">Shell&gt; beeline -u jdbc:hive2://192.168.131.21:10000 -n hadoop -p hadoop
</code></pre>
    </div>

    
    <div class="post-copyright">
  <p class="copyright-item">
    <span class="item-title">文章作者</span>
    <span class="item-content">macd</span>
  </p>
  <p class="copyright-item">
    <span class="item-title">上次更新</span>
    <span class="item-content">2017-10-12</span>
  </p>
  
  
</div>

    
    
<div class="post-reward">
  <input type="checkbox" name="reward" id="reward" hidden />
  <label class="reward-button" for="reward">赞赏支持</label>
  <div class="qr-code">
    
    
      <label class="qr-code-image" for="reward">
        <img class="image" src="/img/reward/wechat-qr-code.png">
        <span>微信打赏</span>
      </label>
    
      <label class="qr-code-image" for="reward">
        <img class="image" src="/img/reward/alipay-qr-code.png">
        <span>支付宝打赏</span>
      </label>
  </div>
</div>

    <footer class="post-footer">
      
        <div class="post-tags">
          
          <a href="/tags/hadoop/">hadoop</a>
          
          <a href="/tags/hive/">hive</a>
          
        </div>

      
      <nav class="post-nav">
        
          <a class="prev" href="/wiki/bigdata/hive-sql/">
            <i class="iconfont icon-left"></i>
            <span class="prev-text nav-default">Hive SQL</span>
            <span class="prev-text nav-mobile">上一篇</span>
          </a>
        
          <a class="next" href="/wiki/oracle/oracle%E5%AF%BC%E5%BA%93/">
            <span class="next-text nav-default">Oracle导库</span>
            <span class="prev-text nav-mobile">下一篇</span>
            <i class="iconfont icon-right"></i>
          </a>
      </nav>
    </footer>
  </article>
        </div>
        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="social-links">
      <a href="mailto:macdfree@163.com" class="iconfont icon-email" title="email"></a>
      <a href="https://github.com/MACDfree" class="iconfont icon-github" title="github"></a>
  <a href="/index.xml" type="application/rss+xml" class="iconfont icon-rss" title="rss"></a>
</div>

<div class="copyright">
  <span class="power-by">
    由 <a class="hexo-link" href="https://gohugo.io">Hugo</a> 强力驱动
  </span>
  <span class="division">|</span>
  <span class="theme-info">
    主题 - 
    <a class="theme-link" href="https://github.com/olOwOlo/hugo-theme-even">Even</a>
  </span>

  <div class="busuanzi-footer">
    <span id="busuanzi_container_site_pv"> 本站总访问量 <span id="busuanzi_value_site_pv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 次 </span>
    <span class="division">|</span>
    <span id="busuanzi_container_site_uv"> 本站总访客数 <span id="busuanzi_value_site_uv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 人 </span>
  </div>

  <span class="copyright-year">
    &copy; 
    
      2014 - 
    2019
    <span class="heart">
      <i class="iconfont icon-heart"></i>
    </span>
    <span class="author">macd</span>
    <span class="division">|</span>
    <span style="margin-left: 5px;">
      <img alt="buildstatus" src="https://travis-ci.org/MACDfree/hugoblog.svg?branch=master">
    </span>
  </span>
</div>
    </footer>

    <div class="back-to-top" id="back-to-top">
      <i class="iconfont icon-up"></i>
    </div>
  </div>
  
<script src="/lib/highlight/highlight.pack.js?v=20171001"></script><script type="text/javascript" src="/lib/jquery/jquery-3.2.1.min.js"></script>
  <script type="text/javascript" src="/lib/slideout/slideout-1.0.1.min.js"></script>
  <script type="text/javascript" src="/lib/fancybox/jquery.fancybox-3.1.20.min.js"></script>
<script type="text/javascript" src="/dist/even.min.js?v=3.1.1"></script>








</body>
</html>
